{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a80479be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://www.boxofficecn.com/boxoffice2010\n",
      "Downloading: http://www.boxofficecn.com/boxoffice2011\n",
      "Downloading: http://www.boxofficecn.com/boxoffice2012\n",
      "Downloading: http://www.boxofficecn.com/boxoffice2013\n",
      "Downloading: http://www.boxofficecn.com/boxoffice2014\n",
      "Downloading: http://www.boxofficecn.com/boxoffice2015\n",
      "Downloading: http://www.boxofficecn.com/boxoffice2016\n",
      "Downloading: http://www.boxofficecn.com/boxoffice2017\n",
      "Downloading: http://www.boxofficecn.com/boxoffice2018\n",
      "Downloading: http://www.boxofficecn.com/boxoffice2019\n"
     ]
    }
   ],
   "source": [
    "#电影数据爬虫\n",
    "import re\n",
    "import pandas as pd\n",
    "import time\n",
    "import urllib.request\n",
    "from lxml.html import fromstring\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "\n",
    "# 下载链接\n",
    "def download(url):\n",
    "    print('Downloading:', url)\n",
    "    request = urllib.request.Request(url)\n",
    "    request.add_header('User-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36') #进行伪装\n",
    "    resp = urllib.request.urlopen(request)\n",
    "    html = resp.read().decode('utf-8')\n",
    "    return html\n",
    "\n",
    "# 待爬取内容\n",
    "name = []\n",
    "year = []\n",
    "Box_office = []\n",
    "\n",
    "# 循环爬取每页内容\n",
    "for k in range(10):\n",
    "    movie_year = 2010+k\n",
    "    url = download('http://www.boxofficecn.com/boxoffice{}'.format(movie_year))\n",
    "    time.sleep(3)   #间隔3s，防止被封禁，爬取是有限制的\n",
    "    tree = fromstring(url)\n",
    "    soup = BeautifulSoup(url)\n",
    "    length_string = soup.find('div',{'class':'entry-content'}).p.get_text()\n",
    "    length = int(re.search('[0-9]{1,3}(?=部)',length_string).group())\n",
    "    for k in range(length):\n",
    "        name.append(soup.find_all('tbody')[0].find_all('td')[4*k+2].get_text())\n",
    "        year.append(movie_year)\n",
    "        Box_office.append(soup.find_all('tbody')[0].find_all('td')[4*k+3].get_text())\n",
    "\n",
    "# 将list转化为dataframe\n",
    "name_pd = pd.DataFrame(name)\n",
    "year_pd = pd.DataFrame(year)\n",
    "Box_office_pd = pd.DataFrame(Box_office)\n",
    "\n",
    "# 三部分拼接起来\n",
    "movie_Box_office_data = pd.concat([name_pd,year_pd,Box_office_pd],axis=1)\n",
    "movie_Box_office_data.columns=['电影','年份','票房']\n",
    "\n",
    "movie_Box_office_data.head()\n",
    "\n",
    "# 数据预处理\n",
    "## 提取数字部分\n",
    "f = lambda x: re.search('[0-9]*(\\.[0-9]*)?',x).group()\n",
    "movie_Box_office_data['票房'] = movie_Box_office_data['票房'].apply(f)\n",
    "## 缺失值填充为0\n",
    "empty = movie_Box_office_data['票房'] == ''\n",
    "movie_Box_office_data.loc[empty,'票房'] = 0\n",
    "## 转化成浮点数\n",
    "movie_Box_office_data['票房'] = movie_Box_office_data['票房'].apply(lambda x: float(x))\n",
    "\n",
    "\n",
    "# 输出\n",
    "outputpath=r'C:\\Users\\Administrator\\PycharmProjects\\pythonProject/movie_box_office.xlsx' #路径\n",
    "movie_Box_office_data.to_csv(outputpath,sep=',',index=False,header=True,encoding='utf_8_sig')\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
